#include "../../include/PageLib.h"
#include "../../include/tinyxml2.h"
#include "../../include/ConfigFile.h"

#include <unistd.h>
#include <regex>
#include <fstream>
#include <iostream>
#include <sstream>

using namespace tinyxml2;
using std::regex;
using std::cout;
using std::endl;
using std::ifstream;
using std::ofstream;
using std::istringstream;



PageLib::PageLib(const string &dirPath)
: _dirScanner(dirPath)
, _invertIndexMaker(_webPagesNew, _invertIndexTable)
, _wordSeg()
{
    loadStopWords();
    createOldPageLib();
    cutRedundantPages();
    makeWebWordsFreq();
    _invertIndexMaker.createTable();
    storeNewPageLib();
    storeInvertIndexTable();
}

void PageLib::createOldPageLib()
{
    vector<string> _filesPath = _dirScanner.getFilesPath();
    XMLElement * channel, *item, *title, *url, *description, *content, *content_encode;
    for(size_t idx = 0; idx < _filesPath.size(); ++idx) {
        XMLDocument doc;
        //cout << _filesPath[idx] << endl;
        if(doc.LoadFile(_filesPath[idx].c_str()) != XML_SUCCESS) {
            perror("doc LoadFile");
            exit(-1);
        }
        channel = doc.RootElement()->FirstChildElement("channel");
        item = channel->FirstChildElement("item");
        while(item) {
            struct RssItem rss;
            title = item->FirstChildElement("title");
            url = item->FirstChildElement("link");
            description = item->FirstChildElement("description");
            content = item->FirstChildElement("content");
            content_encode = item->FirstChildElement("content:encoded");
            
            if(description) {
                string s = mv_tag(description->GetText());
                rss.title = title->GetText();
                rss.url = url->GetText();
                if(s != "") {
                    if(content) {
                        rss.content = mv_tag(content->GetText());
                    } else if(content_encode) {
                        rss.content = mv_tag(content_encode->GetText());
                    } else {
                        rss.content = s;
                    }
                } else {
                    rss.content = rss.title;
                }
                WebPage web(rss);
                //cout << "wordSeg.size():" << sizeof(_wordSeg) << endl;
                //cout << "web.size():" << sizeof(web) << endl;
                //while(1);
                _webPages.emplace_back(web);
            }
            item = item->NextSiblingElement();
        }
    }

    cout << "[create oldpages succese!]" << endl;
}



//存储去重后网页库和偏移库
void PageLib::storeNewPageLib()
{
    size_t initOffset = 0;
    ConfigFile *confPtr = ConfigFile::getInstance();

    string page_path = confPtr->getConf()["ripePagePath"];
    string offset_path = confPtr->getConf()["offsetPath"];



    // if (access(page_path.c_str(), F_OK) != -1) {
    //     cout << "oldripepage.dat已经存在" << endl;
    // } else if(access(offset_path.c_str(), F_OK) != -1) {
    //     cout << "oldoffset.dat已经存在"  << endl;
    // } 
    // else {
        ofstream ofsPage, ofsOffset;
        ofsPage.open(page_path, std::ios::out);
        if(!ofsPage.good()) {
            std::cerr << "ofstream open file error" << endl;
            return;
        }
        ofsOffset.open(offset_path, std::ios::out);
        if(!ofsOffset.good()) {
            std::cerr << "ofstream open file error" << endl;
            return;
        }

        for(auto &file : _webPagesNew) {
            ofsPage << file.getDoc();
            
            size_t length = file.getLength();
            ofsOffset << file.getDocId() << " " << initOffset << " " << length << endl;
            initOffset += length;
        }

        ofsPage.close();
        ofsOffset.close();
    //}   
}

void PageLib::storeInvertIndexTable() 
{
    ConfigFile *confPtr = ConfigFile::getInstance();

    string invertPath = confPtr->getConf()["invertIndexPath"];
    ofstream ofsInvert;
    ofsInvert.open(invertPath, std::ios::out);
    if(!ofsInvert.good()) {
        std::cerr << "ofsInvert open file error" << endl;
        return;
    }
    for (auto &invertIndexElem : _invertIndexTable) {
        ofsInvert << invertIndexElem.first;
        for (auto &docIdElem : invertIndexElem.second) {
            ofsInvert << " " << docIdElem.first << " " << docIdElem.second;
        }
        ofsInvert << endl;
    }

    ofsInvert.close();
}


string PageLib::mv_tag(const string &s)
{
    string res = s;
    regex rx("<[^>]+>|&#8211");
    res = std::regex_replace(res, rx, "");

    // regex rx1("                               ");
    // res = std::regex_replace(res, rx, "");
    // regex rx1("&nbsp;");
    // res = std::regex_replace(res, rx1, "");

    // regex rx2("showPlayer\\(\\{.*?\\}\\);");
    // res = std::regex_replace(res, rx2, "");    
    return res;
}


void PageLib::cutRedundantPages() 
{
    uint64_t u64;
    size_t topN = 5;
    size_t docId = 1;
    for(auto &page : _webPages) {
        PageLibPreprocessor::getInstance()->getSimHasher(page.getDocContent(), topN, u64);
        auto ret = _simHashs.insert(u64);

        if(ret.second) {
            page.setDocId(docId);
            page.makeformatDoc();
            _webPagesNew.emplace_back(page);
            ++docId;
        }
    }
    cout << "[去重完成！]" << endl;
}

void PageLib::loadStopWords() {
    ConfigFile *confPtr = ConfigFile::getInstance();
    string enStopWordsPath = confPtr->getConf()["stopEnPath"];
    string cnStopWordsPath = confPtr->getConf()["stopZhPath"];
    ifstream enIfs;
    enIfs.open(enStopWordsPath, ifstream::in);
    ifstream cnIfs;
    cnIfs.open(cnStopWordsPath, ifstream::in);

    if (!enIfs.good()) {
        perror("getStopWords ifs");
        return;
    }

    if (!cnIfs.good()) {
        perror("getStopWords ifs");
        return;
    }

    string lines, word;

    while (getline(enIfs, lines)) {
        istringstream iss(lines);
        iss >> word;
        _stopWords.insert(word);
    }

    while (getline(cnIfs, lines)) {
        istringstream iss(lines);
        iss >> word;
        _stopWords.insert(word);
    }

    cout << "[load stopWords ok!]" << endl;
    enIfs.close();
    cnIfs.close();

}

void PageLib::makeWebWordsFreq()
{
    for(auto &page : _webPagesNew) {
        page.createWordsFreq(_stopWords, _wordSeg);
    }
    cout << "[makeWebWordsFrq ok!]" << endl;
}